#/usr/bin/env python
#-*-coding:utf-8-*-

"""
关键词匹配
"""

import fool
import jieba
import math
import time
from jieba import posseg

class KeyWordExtractor():
    def __init__(self):
        self.keep_pos = ['n','v','nr', 'nr', 'nt']

    def ners(self, text):
        entity = []
        ners = fool.ner(text)[0]
        for item in ners:
            entity.append(item[-1])
        return entity

    def extract(self, text):
        keywords = []
        if not text:
            return keywords
        text = text.lower()
        ners = self.ners(text)

        words = list(posseg.cut(text))
        words = [item.word for item in words if item.flag in self.keep_pos]
        keywords.extend(ners)
        keywords.extend(words)
        return keywords


import data
import btm
import json
url = "http://localhost:18080/topic"

def keyword_match():
    extractor = KeyWordExtractor()
    outf = open("topic_match.csv", 'w')

    all_data = data.Data()
    comment_dict = {}
    comment_dict_prob = {}
    start = time.time()
    for i, comment in enumerate(all_data.comments):
        comment_dict[i] = []
        comment_dict_prob[i] = []
        try:
            topic = btm.get_lda_topic(comment, url)
        except Exception as e:
            print(e)
            continue

        topics = topic.get("topics", [])
        for topic in topics:
            topic_id = topic.get("topicId", 0)
            prob = topic.get("prob", 0.0)
            if prob>0.1:
                comment_dict[i].append(topic_id)
                comment_dict_prob[i].append(prob)

    print(time.time() - start)
    print(len(comment_dict))
    # for item in comment_dict.items():
    #     print(item)

    for i, video in enumerate(all_data.videos):
        title = video[1]
        topic = btm.get_lda_topic(title, url)
        topics = topic.get("topics", [])

        v_t = []
        v_p = []
        for t in topics:
            topic_id = t.get("topicId")
            prob = t.get("prob", 0.0)
            if prob>0.1:
               v_t.append(topic_id)
               v_p.append(prob)

        if not v_t:
            continue

        max_dis = 0.0
        max_index = -1

        for c_id, v in comment_dict.items():
            join_s = set(v_t) & set(v)

            if not join_s:
                continue
            tmps = []
            for j in join_s:
                v_prob = v_p[v_t.index(j)]
                c_prob = comment_dict_prob[c_id][v.index(j)]
                tmps.append(abs(c_prob))

            total_sc = sum(tmps)
            if total_sc> max_dis:
                max_dis = total_sc
                max_index = c_id

        if max_index>0:
            outf.write("\t".join(video)+"\t"+all_data.comments[max_index] + "\n")


                # print(v_prob, c_prob)
                # if abs(v_prob-c_prob) < 0.3:
                #     outf.write("\t".join(video)+"\t"+all_data.comments[c_id] + "\n")

            # if len(join_s) >1:
                # video.append(all_data.comments[c_id])
                # outf.write("\t".join(video)+"\t"+all_data.comments[c_id] + "\n")

if __name__ == '__main__':
    # extractor = KeyWordExtractor()
    # extractor.extract("Gaga管弦乐合唱团最新演绎")
    keyword_match()